
from langchain_community.document_loaders import PyPDFLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_text_splitters import CharacterTextSplitter

loader = PyPDFLoader("data/中小企业数字化转型指南.pdf")
pages = loader.load_and_split()

# fixed_text = pages[0].page_content.replace("\n", " ")
# print(fixed_text)
fixed_text = pages[0].page_content
print(len(fixed_text))

c_splitter = CharacterTextSplitter(
    chunk_size=200,
    chunk_overlap=20,
    separator='\n',
    is_separator_regex=True
)
# docs = c_splitter.split_documents(pages)
# print(type(docs[0]))
# print(len(docs))
# print(docs[:5])
texts = c_splitter.split_text(fixed_text)
print(len(texts))
for t in texts:
    print(t)
    print('-'*8)

print('#'*8)
r_splitter = RecursiveCharacterTextSplitter(
    chunk_size=250,
    chunk_overlap=20,
    # separators=['\n'], # 不需要设置，默认为["\n\n", "\n", " ", ""]
    # is_separator_regex=True
)

texts = r_splitter.split_text(fixed_text)
print(len(texts))
for t in texts:
    print(t)
    print('*'*8)
